package edu.hit.pact.pgse.crawler.bbs;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.logging.Log;

import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.crawler.download.WebPageFetcher;
import edu.hit.pact.pgse.crawler.store.WebPageStore;
import edu.hit.pact.pgse.crawler.util.Anchor;
import edu.hit.pact.pgse.crawler.util.DuplicateUrlFilter;

public class BBSCrawlerIndexFetchProcessor extends AbstractBBSFetchorThread {

	//private URL homepage;
	private static final int PAGENUM = 2000; 
	private DuplicateUrlFilter indexDuplicateUrlFilter;//避免index页的重复
	protected DuplicateUrlFilter aritcleDuplicateUrlFilter; //避免帖子的重复
	private boolean isShedule;

	public BBSCrawlerIndexFetchProcessor(Log logger, WebPageFetcher fetcher,WebPageStore webPageStore,
			ConcurrentLinkedQueue<URL> articleUrlQueue, String threadName,URL homepage) {
		
		super(logger, fetcher, webPageStore, articleUrlQueue, threadName);
		this.homepage = homepage;
		this.indexDuplicateUrlFilter = new DuplicateUrlFilter();
		this.aritcleDuplicateUrlFilter = new DuplicateUrlFilter();
	}
	
	public BBSCrawlerIndexFetchProcessor(Log logger, WebPageFetcher fetcher,WebPageStore webPageStore,
			ConcurrentLinkedQueue<URL> articleUrlQueue, String threadName){
		super(logger, fetcher, webPageStore, articleUrlQueue, threadName);
		this.indexDuplicateUrlFilter = new DuplicateUrlFilter();
		this.aritcleDuplicateUrlFilter = new DuplicateUrlFilter();
	}

	public BBSCrawlerIndexFetchProcessor(Log logger, WebPageFetcher fetcher,WebPageStore webPageStore,
			ConcurrentLinkedQueue<Anchor> articleAnchorQueue, String threadName,URL homepage,AtomicBoolean isFinished,
			boolean isShedule) {
		
		super(logger, fetcher, webPageStore, articleAnchorQueue, threadName,isFinished);
		this.homepage = homepage;
		this.indexDuplicateUrlFilter = new DuplicateUrlFilter();
		this.aritcleDuplicateUrlFilter = new DuplicateUrlFilter();
		this.isShedule = isShedule;
	}
	

	@Override
	public void mainRun() throws MalformedURLException ,InterruptedException{

		TianYaBBSURL topicUrl = new TianYaBBSURL(homepage);
		topicUrl.fllter();
		String subitem = topicUrl.getSubitem();
		int pageno = topicUrl.getPageno();
		int pageNum = 0;
//		String path = Utilities.getDay() + "/" + homepage.getHost() + "/"
//				+ subitem + "/";
		//Utilities.createdDir(path);
		
		BBSUnit bbsUnit = new BBSUnit();
		URL url = homepage;
		
		this.aritcleDuplicateUrlFilter.contains("http://www.tianya.cn/publicforum/content/worldlook/1/200740.shtml");
		this.aritcleDuplicateUrlFilter.contains("http://www.tianya.cn/publicforum/content/worldlook/1/161233.shtml");
		
		boolean isWakeUp = false;
		while (pageNum++ < PAGENUM && url != null) {
			
			if(shedule(isShedule)){
				this.fetcher = buildWebPageFetcher();
			}
//			while(this.articleUrlQueue.size() > 200){
//				isWakeUp = true;
//				sleep(300000);
//			}
			
			while(this.articleAnchorQueue.size() > 200){
				isWakeUp = true;
				sleep(300000);
			}
			
			if(isWakeUp){
				this.fetcher = buildWebPageFetcher();
				isWakeUp = false;
			}
			
			if(this.indexDuplicateUrlFilter.contains(url.toExternalForm())){
				logger.info(this.getName() + " maybe in deap loop,so give up to continue:"+url.toExternalForm());
				break;
			}
			
			RawWebPage indexPage = null;
			while (null == indexPage) {
				indexPage = this.fetcher.fetchURL(url, 3, null, null, null);
				if (null == indexPage) {
					logger.info(this.getName()+" failed to download  and will try again:"
							+ url.toString());
					continue;
				}
			}
//			String fileName = pageno+".txt";
//			this.save(fileName, indexPage.getPageRawContent());
//			logger.error("last_modified :" + indexPage.getLastModified());
			List<String> interLinkList = indexPage.getCommentLinks();
			List<Anchor> interAnchorList = indexPage.getAnchorList(); 
			boolean flag  = false;
			
			
			for (Anchor anchor : interAnchorList) {
				URL tmpUrl = anchor.getUrl();
				if (bbsUnit.isArticleURL(tmpUrl,subitem) && !this.aritcleDuplicateUrlFilter.contains(tmpUrl.toExternalForm())) {
					this.articleAnchorQueue.add(anchor);
				} else if (bbsUnit.isNextPageUrl(tmpUrl,subitem,pageno)) {
					url = tmpUrl;
					flag = true;
				}else {
					//do nothing
				}
			}
			
//			for (String interLink : interLinkList) {
//				URL tmpUrl = new URL(interLink);
//				if (bbsUnit.isArticleURL(tmpUrl,subitem) && !this.aritcleDuplicateUrlFilter.contains(tmpUrl.toExternalForm())) {
//					articleUrlQueue.add(tmpUrl);
//				} else if (bbsUnit.isNextPageUrl(tmpUrl,subitem,pageno)) {
//					url = tmpUrl;
//					flag = true;
//				}else {
//					//do nothing
//				}
//			}
			
			if(!flag){
				url = null;
			}
			
			this.webPageStore.storeWebPage(indexPage, subitem+"/",pageno+"",true);
			this.successedPageNum ++;
			pageno ++;
			
			if(null != url){
			logger.info(this.getName()+" next page url is : " + url.toExternalForm());
			}else{
				logger.info(this.getName()+" next page url is null");
			}
				
		}
		isFinished.set(true);

	}
	
	//just for test
	private void save(String fileName,String content){
		FileOutputStream stream = null;
		try {
			stream = new FileOutputStream(fileName);
		} catch (FileNotFoundException e1) {
			
			logger.error("maybe too many links and cannot create new folder");
	
		}
		try {
			ObjectOutputStream out = new ObjectOutputStream(stream);
			out.writeObject(content);
			out.close();

		} catch (IOException e) {
			logger.error("store object meets exception", e);
		}// end try
	}
}
